import json
import time

import scrapy
from spidertools.utils.time_utils import get_current_date

from commonresources.spider_items.jiangsu.nantong.items import NanTongGongGongZiYuanJiaoYiWangItem
from spidertools.common_pipeline.base_item import convert_dict


class NanTongGongGongZiYuanJiaoYiWangSpider(scrapy.Spider):
    """
        南通市公共资源交易网  交易信息列表页获取爬虫
        （http://zfcg.nantong.gov.cn/）
    """
    name = 'NanTongGongGongZiYuanJiaoYiWang'
    name_zh = '南通市公共资源交易网'
    province = "江苏"
    city = '南通'
    allowed_domains = ['zfcg.nantong.gov.cn']
    start_urls = [
        'http://zfcg.nantong.gov.cn/services/XzsJsggWebservice/getListByCount?response=application/json&&categorynum=&bianhao=&xmmc=']

    def __init__(self, full_dose=False):
        self.browser_cookie = {}
        self.page = -1
        self.convert_dict = convert_dict
        self.full_dose = full_dose

        super().__init__()

    def get_page_headers(self):
        headers = {
            "Accept": "application/json, text/javascript, */*; q=0.01",
            "Accept-Encoding": "gzip, deflate",
            "Accept-Language": "zh-CN,zh;q=0.9",
            "Connection": "keep-alive",
            "Host": "zfcg.nantong.gov.cn",
            "Referer": "http://zfcg.nantong.gov.cn/jyxx/tradeInfo.html",
            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.25 Safari/537.36 Core/1.70.3775.400 QQBrowser/10.6.4209.400", }
        return headers

    def parse(self, response):
        page_obj = json.loads(response.text)
        self.page = int(page_obj['return']) // 15 + 1
        next_page_url = f'http://zfcg.nantong.gov.cn/services/XzsJsggWebservice/getList?response=application/json&pageIndex=0&pageSize=15&&categorynum=&bianhao=&xmmc='
        yield scrapy.Request(url=next_page_url, headers=self.get_page_headers(), cookies=self.browser_cookie,
                             meta={'page': 0,'need_break':False},
                             callback=self.handle_response)

    def handle_response(self, response):
        """处理请求返回的数据"""
        obj_json = json.loads(response.text)
        obj_json_page = json.loads(obj_json['return'])
        rows_info_list = obj_json_page['Table']
        for row in rows_info_list:
            item = dict()
            item['release_time'] = row['postdate']
            if not self.full_dose and item['release_time'] != get_current_date() and item['release_time'] != "2020-10-28":
                response.meta["need_break"] = True
            item['construction_type'] = row['jyfl']
            item['project_area'] = row['city']
            item['origin_url'] = "http://zfcg.nantong.gov.cn" + row['href']
            item['announcement_title'] = row['title']
            item['project_transaction_means'] = row['jyfs']
            yield scrapy.Request(url=item['origin_url'], meta=item, cookies=self.browser_cookie,
                                 callback=self.handle_next_html)
        if not response.meta["need_break"]:
            page = response.request.meta['page']
            if page < self.page:
                page += 1
                next_page_url = f'http://zfcg.nantong.gov.cn/services/XzsJsggWebservice/getList?response=application/json&pageIndex={page}&pageSize=15&&categorynum=&bianhao=&xmmc='
                yield scrapy.Request(url=next_page_url, headers=self.get_page_headers(), cookies=self.browser_cookie,
                                     meta={'page': page,'need_break':False},
                                     callback=self.handle_response)

    def handle_next_html(self, response):
        item = NanTongGongGongZiYuanJiaoYiWangItem()
        item['project_area'] = response.request.meta['project_area']
        item['construction_type'] = response.request.meta['construction_type']
        item['release_time'] = response.request.meta['release_time']
        item['origin_url'] = response.request.meta['origin_url']
        item['announcement_title'] = response.request.meta['announcement_title']
        item['project_transaction_means'] = response.request.meta['project_transaction_means']
        item['html'] = response.text
        item['source_type'] = self.name_zh
        item['is_parsed'] = 0
        item['province'] = self.province
        item['city'] = self.city
        yield item
